Imports¶
In [5]:
# Import Modules
import os
import pandas as pd
import numpy as np
from itertools import chain
import cv2
import split_folders
# Plotting
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
plt.style.use('ggplot')
import seaborn as sns
# Metrics
from sklearn.metrics import confusion_matrix, roc_curve,auc, classification_report
# Deep Learning
import tensorflow as tf
print(tf.__version__)
from tensorflow.keras import Model
from tensorflow.keras.layers import Flatten, Dense, Dropout, GlobalAveragePooling2D
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.models import load_model
from tensorflow.keras.optimizers import SGD, Adam
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.vgg19 import preprocess_input as vgg19_preprocess_input
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.resnet50 import preprocess_input as resnet50_preprocess_input
from tensorflow.keras.applications.mobilenet import MobileNet
from tensorflow.keras.applications.mobilenet import preprocess_input as mobilenet_preprocess_input
from tensorflow.keras.preprocessing import image
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing.image import array_to_img
The dataset¶
The dataset used for training are 100,000 HE image tiles from 86 Colorectal Cancer tissue slides. The images are classified into 9 different tissue types: ADI (adipose tissue), BACK (background), DEB (debris), LYM (lymphocytes), MUC (mucus), MUS (smooth muscle), NORM (normal colon mucosa), STR (cancer-associated stroma), and TUM (colorectal adenocarcinoma epithelium) and are organized in 9 different folders.
In [5]:
base_dir = "/floyd/input/colorectal_cancer"
input_ = os.path.join(base_dir,"NCT-CRC-HE-100K")
# split data into training, vlaidation and testing sets
split_folders.ratio(input_, 'output', seed = 101, ratio=(0.8, 0.1, 0.1))
data_dir = os.path.join('/floyd/home','output')
# Define train, valid and test directories
train_dir = os.path.join(data_dir, 'train')
valid_dir = os.path.join(data_dir, 'val')
test_dir = os.path.join(data_dir, 'test')
In [6]:
os.listdir('output')
Out[6]:
In [8]:
# Plot number of samples for each class in train_dir
# Path to all three directories
train_dir = os.path.join(data_dir, 'train')
valid_dir = os.path.join(data_dir, 'val')
test_dir = os.path.join(data_dir, 'test')
# Tissue classes list
class_ = os.listdir(train_dir)
# Instantiate empty lists to add the number of samples
trn_num_samples = []
val_num_samples = []
test_num_samples = []
# Count the number of samples in train_dir
for dir_ in [train_dir, valid_dir, test_dir]:
if dir_ == train_dir:
for dir_ in os.listdir(dir_):
trn_num_samples.append(len(os.listdir(os.path.join(train_dir, dir_))))
elif dir_ == valid_dir:
for dir_ in os.listdir(dir_):
val_num_samples.append(len(os.listdir(os.path.join(valid_dir, dir_))))
else:
for dir_ in os.listdir(dir_):
test_num_samples.append(len(os.listdir(os.path.join(test_dir, dir_))))
# Make a dataframe with the computed stats
# Step 1: Make a dictionary and populate the dictionary
stats = {}
stats['Tissue_type'] = class_
stats['Train'] = trn_num_samples
stats['Valid'] = val_num_samples
stats['Test'] = test_num_samples
# Step 2: Make a dataframe and sort its values based on train data
df = pd.DataFrame(stats).sort_values(['Train'], ascending=True)
# Plot the data
fig, (ax1, ax2, ax3) = plt.subplots(nrows=1, ncols=3, figsize=(15,6), sharey=True)
fig.suptitle("Distribution of tissue images", fontsize=24)
df.plot(kind='barh', x="Tissue_type", y='Train', ax = ax1, fontsize=12)
ax1.set_xlabel('Number of samples', fontsize=16)
ax1.set_ylabel('Tissue type', fontsize=16)
df.plot(kind='barh', x="Tissue_type", y='Valid', ax = ax2, fontsize=12)
ax2.set_xlabel('Number of samples', fontsize=16)
df.plot(kind='barh', x="Tissue_type", y='Test', ax = ax3, fontsize=12)
ax3.set_xlabel('Number of samples', fontsize=16)
Out[8]:
Data Augmentation and EDA¶
Data augmentation is a strategy that adds noise and increase the diversity of the training data. It is a form of regularization and generalizes tbe model.
The method purposely perturbs training examples, changing their appearance slightly, before passing them into the network for training. This partially alleviates the need to gather more training data.
Initialize each of our generators with data augmentation techniques such as horizontal flipping, vertical flipping, shear, rotation and shift are commonly used to train large neural networks.
In [9]:
# Data Augmentation
train_datagen = ImageDataGenerator(rescale=1./255.,
horizontal_flip=True,
vertical_flip=True,
shear_range=0.4,
width_shift_range=0.25,
height_shift_range=0.25,
rotation_range=45,
fill_mode='nearest')
valid_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)
In [2]:
# Train ImageDataGenerator
train_generator = train_datagen.flow_from_directory(train_dir,
batch_size = 64,
target_size = (224,224),
class_mode = 'categorical',
shuffle=True,
seed=42,
color_mode='rgb')
valid_generator = valid_datagen.flow_from_directory(valid_dir,
batch_size=64,
target_size=(224,224),
class_mode='categorical',
shuffle=True,
seed=42,
color_mode='rgb')
test_generator = test_datagen.flow_from_directory(test_dir,
batch_size=1,
target_size=(224,224),
class_mode='categorical',
shuffle=False,
seed=42,
color_mode='rgb')
Plot images for each class¶
In [13]:
# Test data
filenames = test_generator.filenames
nb_samples = len(filenames)
In [11]:
# Get model class_labels and inverted class_labels as class_names
class_labels = train_generator.class_indices
class_names = {value:key for key,value in class_labels.items()}
# Tissue types dictionary mapping class names with full names
tissue_types = {"ADI": "Adipose tissue",
"BACK": "Background",
"DEB": "Debris",
"LYM": "Lymphocyte aggregates",
"MUC": "Mucus",
"MUS": "Muscle",
"NORM": "Normal mucosa",
"STR": "Stroma",
"TUM": "Tumor epithelium"}
# List of images to plot: 4 images from each class in the training data
images_to_plot = [os.path.join(train_dir, subdir_, fname) for subdir_ in os.listdir(train_dir) for fname in os.listdir(os.path.join(train_dir, subdir_))[:4]]
fig = plt.figure(figsize=(30,60))
fig.subplots_adjust(top=0.88)
columns = 4
rows = 9
for i in range(1, columns*rows+1):
fig.add_subplot(rows, columns, i)
plt.imshow(mpimg.imread(images_to_plot[i-1]))
plt.axis('off')
title = images_to_plot[i-1].split('/')[-2]
plt.title("True Label: {}".format(tissue_types[title]), fontsize=28)
plt.tight_layout()
plt.show()
VGG19 models¶
We will use a pre-trained VGG19 model for this multiclass classification task with weights from ImageNet trained on a very large, very popular dataset used for image classification and other vision tasks.
VGG19 training cycle #1¶
In [12]:
# Instantiate the vgg19 model without the top classifier.
base_model = VGG19(input_shape=(224, 224, 3), weights='imagenet', include_top = False)
# Add a classifier to the convolution block classifier
x = base_model.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
# Define the model
model_vgg19_01 = Model(base_model.inputs, output)
# Freeze all layers in the convolution block. We don't want to train these weights yet.
for layer in base_model.layers:
layer.trainable=False
#model_vgg19_01.summary()
# Call Backs
filepath = 'vgg19_base_model_wt.h5'
es = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
# Compile a model
model_vgg19_01.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.0001), metrics=["accuracy"])
In [22]:
# Model fitting
history_01 = model_vgg19_01.fit_generator(
train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es, cp],
validation_data = valid_generator)
In [23]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_vgg19_01.save_weights(filepath='model_weights/vgg19_base_model_wt.h5', overwrite=True)
Model Evaluation: training round #1¶
In [24]:
# Load the saved model
model_vgg19_01.load_weights('vgg19_base_model_wt.h5')
# Evaluate the model on the hold out validation and test datasets
vgg_val_eval_01 = model_vgg19_01.evaluate_generator(valid_generator)
vgg_test_eval_01 = model_vgg19_01.evaluate_generator(test_generator)
print('Validation loss: {}'.format(vgg_val_eval_01[0]))
print('Validation accuracy: {}'.format(vgg_val_eval_01[1]))
print('Test loss: {}'.format(vgg_test_eval_01[0]))
print('Test accuracy: {}'.format(vgg_test_eval_01[1]))
# Predict probabilities
vgg_predictions_01 = model_vgg19_01.predict_generator(test_generator,steps = nb_samples,verbose=1)
# Predict labels
vgg_pred_labels_01 = np.argmax(vgg_predictions_01, axis=1)
# Classification Report
print('|'+'-'*75+'|')
print('|-------------------Classification Report: Training Cycle #1----------------|')
print('|'+'-'*75+'|')
print(classification_report(test_generator.classes, vgg_pred_labels_01,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
In [25]:
# Plot performance of vgg19 base model
fig, (ax1,ax2) = plt.subplots(nrows=1, ncols=2, figsize=(12,6))
fig.suptitle("VGG19 Base Model Training", fontsize=20)
max_epoch = len(history_01.history['acc'])+1
epochs_list = list(range(1, max_epoch))
ax1.plot(epochs_list, history_01.history['acc'], color='b', linestyle='-', label='Training Data')
ax1.plot(epochs_list, history_01.history['val_acc'], color='r', linestyle='-', label ='Validation Data')
ax1.set_title('Training Accuracy', fontsize=14)
ax1.set_xlabel('Epochs', fontsize=14)
ax1.set_ylabel('Accuracy', fontsize=14)
ax1.legend(frameon=False, loc='lower center', ncol=2)
ax2.plot(epochs_list, history_01.history['loss'], color='b', linestyle='-', label='Training Data')
ax2.plot(epochs_list, history_01.history['val_loss'], color='r', linestyle='-', label ='Validation Data')
ax2.set_title('Training Loss', fontsize=14)
ax2.set_xlabel('Epochs', fontsize=14)
ax2.set_ylabel('Loss', fontsize=14)
ax2.legend(frameon=False, loc='upper center', ncol=2)
Out[25]:
Fine tuning VGG19 model Round 2¶
In [26]:
# Construct VGG19 model without the classifer and weights trained on imagenet data
base_model_02 = VGG19(input_shape=(224, 224, 3),
include_top = False)
x = base_model_02.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_vgg19_02 = Model(base_model_02.inputs, output)
# Load weights from the trained base model
model_vgg19_02.load_weights('vgg19_base_model.h5')
# Freeze layers upto the 19th layer.
for layer in model_vgg19_02.layers[:19]:
layer.trainable = False
# Call Backs
filepath = 'vgg19_model_wt_ft_01.h5'
es = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
#print(model_vgg19_02.summary())
model_vgg19_02.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.0001), metrics=["accuracy"])
In [41]:
# Model fitting
history_02 = model_vgg19_02.fit_generator(
train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es, cp],
validation_data = valid_generator)
In [42]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_vgg19_02.save_weights(filepath='model_weights/vgg19_model_wt_ft_01.h5', overwrite=True)
In [44]:
# Plot performance of vgg19 finetune model
fig, (ax1,ax2) = plt.subplots(nrows=1, ncols=2, figsize=(12,6))
fig.suptitle("VGG19 Fine Tuning (Unfreeze 2 Conv Layers)", fontsize=20)
max_epoch = len(history_02.history['acc'])+1
epochs_list = list(range(1, max_epoch))
ax1.plot(epochs_list, history_02.history['acc'], color='b', linestyle='-', label='Training Data')
ax1.plot(epochs_list, history_02.history['val_acc'], color='r', linestyle='-', label ='Validation Data')
ax1.set_title('Training Accuracy', fontsize=14)
ax1.set_xlabel('Epochs', fontsize=14)
ax1.set_ylabel('Accuracy', fontsize=14)
ax1.legend(frameon=False, loc='lower center', ncol=2)
ax2.plot(epochs_list, history_02.history['loss'], color='b', linestyle='-', label='Training Data')
ax2.plot(epochs_list, history_02.history['val_loss'], color='r', linestyle='-', label ='Validation Data')
ax2.set_title('Training Loss', fontsize=14)
ax2.set_xlabel('Epochs', fontsize=14)
ax2.set_ylabel('Loss', fontsize=14)
ax2.legend(frameon=False, loc='upper center', ncol=2)
Out[44]:
Model Evaluation: training round #2¶
In [27]:
# Load the saved model
model_vgg19_02.load_weights('vgg19_model_wt_ft_01.h5')
# Evaluate the model on the hold out validation and test datasets
vgg_val_eval_02 = model_vgg19_02.evaluate_generator(valid_generator)
vgg_test_eval_02 = model_vgg19_02.evaluate_generator(test_generator)
print('Validation loss: {0:.3f}'.format(vgg_val_eval_02[0]))
print('Validation accuracy: {0:.3f}'.format(vgg_val_eval_02[1]))
print('Test loss: {0:.3f}'.format(vgg_test_eval_02[0]))
print('Test accuracy: {0:.3f}'.format(vgg_test_eval_02[1]))
# Predict probabilities
vgg_predictions_02 = model_vgg19_02.predict_generator(test_generator,steps = nb_samples,verbose=1)
# Predict labels
vgg_pred_labels_02 = np.argmax(vgg_predictions_02, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|------------Classification Report: VGG19 Training Cycle #2-----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, vgg_pred_labels_02,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
Fine Tuning the Entire VGG19 Network¶
In [28]:
# Build VGG19
base_model_03 = VGG19(input_shape=(224, 224, 3),
include_top = False)
x = base_model_03.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_vgg19_03 = Model(base_model_03.inputs, output)
# Load the weights saved after the first round of fine tuning
model_vgg19_03.load_weights('vgg19_model_wt_ft_01.h5')
# Callbacks
filepath = 'vgg19_model_finetuned.h5'
cp = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.0001)
model_vgg19_03.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.0001), metrics=["accuracy"])
In [17]:
# Model Fitting
history_03 = model_vgg19_03.fit_generator(
train_generator,
steps_per_epoch=225,
epochs=25,
callbacks = [learning_rate_reduction, cp],
validation_data = valid_generator)
In [18]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_vgg19_03.save_weights(filepath='model_weights/vgg19_model_finetuned.h5', overwrite=True)
In [19]:
model_vgg19_03.save("vgg19_model_finetuned.h5")
Model Evaluation: final model¶
In [29]:
# Load the saved model
model_vgg19_03.load_weights('vgg19_model_finetuned.h5')
# Evaluate the model on the hold out validation and test datasets
vgg_val_eval_03 = model_vgg19_03.evaluate_generator(valid_generator)
vgg_test_eval_03 = model_vgg19_03.evaluate_generator(test_generator)
print('Validation loss: {0:.3f}'.format(vgg_val_eval_03[0]))
print('Validation accuracy: {0:.3f}'.format(vgg_val_eval_03[1]))
print('Test loss: {0:.3f}'.format(vgg_test_eval_03[0]))
print('Test accuracy: {0:.3f}'.format(vgg_test_eval_03[1]))
# Predict probabilities
vgg_predictions_03 = model_vgg19_03.predict_generator(test_generator,steps = nb_samples, verbose=1)
# Predict labels
vgg_pred_labels_03 = np.argmax(vgg_predictions_03, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|------------Classification Report: VGG19 Training Cycle #3-----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, vgg_pred_labels_03,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
Plot Confusion Matrices of all the training rounds¶
In [33]:
# Confusion Matrices
vgg_conf_mat_01 = pd.DataFrame(confusion_matrix(test_generator.classes, vgg_pred_labels_01),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
vgg_conf_mat_02 = pd.DataFrame(confusion_matrix(test_generator.classes, vgg_pred_labels_02),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
vgg_conf_mat_03 = pd.DataFrame(confusion_matrix(test_generator.classes, vgg_pred_labels_03),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
# Plotting Confusion Matrices
sns.set(font_scale=1.2)
fig, (ax1,ax2, ax3) = plt.subplots(nrows=3, ncols=1, figsize=(7,35))
#ax1
sns.heatmap(vgg_conf_mat_01, ax=ax1, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 10})
ax1.set_ylabel("Actual Label", fontsize=20)
ax1.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="VGG19 Training Round #1 \nAccuracy Score: {0:.3f}".format(vgg_test_eval_01[1])
ax1.set_title(all_sample_title, size=24)
ax1.set_ylim(len(vgg_conf_mat_01)-0.1, -0.1)
#ax2
sns.heatmap(vgg_conf_mat_02, ax=ax2, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 10})
ax2.set_ylabel("Actual Label", fontsize=20)
ax2.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="VGG19 Training Round #2 \nAccuracy Score: {0:.3f}".format(vgg_test_eval_02[1])
ax2.set_title(all_sample_title, size=24)
ax2.set_ylim(len(vgg_conf_mat_02)-0.1, -0.1)
#ax3
sns.heatmap(vgg_conf_mat_03, ax=ax3, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 10})
ax3.set_ylabel("Actual Label", fontsize=20)
ax3.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="VGG19 Training Round#3 \nAccuracy Score: {0:.3f}".format(vgg_test_eval_03[1])
ax3.set_title(all_sample_title, size=24)
ax3.set_ylim(len(vgg_conf_mat_03)-0.1, -0.1)
Out[33]:
In [3]:
from itertools import chain
# training_accuracy
training_accuracy = []
training_accuracy.append(history_01.history['acc'])
training_accuracy.append(history_02.history['acc'])
training_accuracy.append(history_03.history['acc'])
training_accuracy_ = list(itertools.chain(*training_accuracy))
# training_loss
training_loss = []
training_loss.append(history_01.history['loss'])
training_loss.append(history_02.history['loss'])
training_loss.append(history_03.history['loss'])
training_loss_ = list(itertools.chain(*training_loss))
# validation_accuracy
validation_accuracy = []
validation_accuracy.append(history_01.history['val_acc'])
validation_accuracy.append(history_02.history['val_acc'])
validation_accuracy.append(history_03.history['val_acc'])
validation_accuracy_ = list(itertools.chain(*validation_accuracy))
# validation_loss
validation_loss = []
validation_loss.append(history_01.history['val_loss'])
validation_loss.append(history_02.history['val_loss'])
validation_loss.append(history_03.history['val_loss'])
validation_loss_ = list(itertools.chain(*validation_loss))
training_metrics_df = pd.DataFrame({'training_accuracy': training_accuracy_,
'training_loss': training_loss_,
'validation_accuracy': validation_accuracy_,
'validation_loss': validation_loss_})
training_metrics_df.to_csv('training_metrics_03.csv', index=False)
In [20]:
# Plot performance of vgg19 finetune model
fig, (ax1,ax2) = plt.subplots(nrows=1, ncols=2, figsize=(12,6))
fig.suptitle("VGG19 Fine Tuning (Unfreeze All Layers)", fontsize=20)
max_epoch = len(history_03.history['acc'])+1
epochs_list = list(range(1, max_epoch))
ax1.plot(epochs_list, history_03.history['acc'], color='b', linestyle='-', label='Training Data')
ax1.plot(epochs_list, history_03.history['val_acc'], color='r', linestyle='-', label ='Validation Data')
ax1.set_title('Training Accuracy', fontsize=14)
ax1.set_xlabel('Epochs', fontsize=14)
ax1.set_ylabel('Accuracy', fontsize=14)
ax1.legend(frameon=False, loc='lower center', ncol=2)
ax2.plot(epochs_list, history_03.history['loss'], color='b', linestyle='-', label='Training Data')
ax2.plot(epochs_list, history_03.history['val_loss'], color='r', linestyle='-', label ='Validation Data')
ax2.set_title('Training Loss', fontsize=14)
ax2.set_xlabel('Epochs', fontsize=14)
ax2.set_ylabel('Loss', fontsize=14)
ax2.legend(frameon=False, loc='upper center', ncol=2)
Out[20]:
MobileNet Models¶
Training cycle #1¶
In [34]:
# Create the base model from the pre-trained model MobileNet V2
base_model_mobilenetv2_01 = MobileNet(input_shape=(224,224,3),
include_top=False)
for layer in base_model_mobilenetv2_01.layers:
layer.trainable=False
# Add the top classification block
x = base_model_mobilenetv2_01.output
flat = GlobalAveragePooling2D()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_mobilenet_01 = Model(base_model_mobilenetv2_01.inputs, output)
model_mobilenet_01.summary()
# Call Backs
filepath = 'mobilenet_base_model_wt.h5'
es_01 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_01 = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
model_mobilenet_01.compile(loss="categorical_crossentropy", optimizer=Adam(lr = 0.0004), metrics=["accuracy"])
In [10]:
mobilenetv2_history_01 = model_mobilenet_01.fit_generator(train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es_01, cp_01],
validation_data = valid_generator)
In [13]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_mobilenet_01.save_weights(filepath='model_weights/mobilenet_base_model_wt.h5',
overwrite=True)
MobileNet Model Evaluation - training cycle #1¶
In [35]:
# Load the saved model
model_mobilenet_01.load_weights('mobilenet_base_model_wt.h5')
# Evaluate the model on the hold out validation and test datasets
mn_val_eval_01 = model_mobilenet_01.evaluate_generator(valid_generator)
mn_test_eval_01 = model_mobilenet_01.evaluate_generator(test_generator)
print('Validation loss: {0:.3f}'.format(mn_val_eval_01[0]))
print('Validation accuracy: {0:.3f}'.format(mn_val_eval_01[1]))
print('Test loss: {0:.3f}'.format(mn_test_eval_01[0]))
print('Test accuracy: {0:.3f}'.format(mn_test_eval_01[1]))
# Prediction Probabilities
mn_predictions_01 = model_mobilenet_01.predict_generator(test_generator,steps = nb_samples, verbose=1)
# Prediction Labels
mn_pred_labels_01 = np.argmax(mn_predictions_01, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: MobileNet Training Cycle #1-----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, mn_pred_labels_01,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
In [126]:
# Prediction Probabilities
mn_predictions_01 = model_mobilenet_01.predict_generator(test_generator,steps = nb_samples, verbose=1)
# Prediction Labels
mn_pred_labels_01 = np.argmax(mn_predictions_01, axis=1)
In [47]:
# Classification Report
print(classification_report(test_generator.classes, mn_pred_labels_01,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
MobileNet - training cycle #2¶
In [36]:
# Build a model
base_model_mobilenet_02 = MobileNet(input_shape=(224, 224, 3),
include_top = False)
x = base_model_mobilenet_02.output
flat = GlobalAveragePooling2D()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_mobilenet_02= Model(base_model_mobilenet_02.inputs, output)
# Load weights from the previous traning session
model_mobilenet_02.load_weights('model_weights/mobilenet_base_model_wt.h5')
# Freeze layers
for layer in model_mobilenet_02.layers[:82]:
layer.trainable=False
# Call Backs
filepath = 'mobilenet_model_02_wt.h5'
es_02 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_02 = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
# compile a model
model_mobilenet_02.compile(loss="categorical_crossentropy", optimizer=Adam(lr = 0.0004), metrics=["accuracy"])
In [19]:
mobilenetv2_history_02 = model_mobilenet_02.fit_generator(train_generator,
steps_per_epoch=225,
epochs=25,
callbacks = [es_02, cp_02],
validation_data = valid_generator)
In [23]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_mobilenet_02.save_weights(filepath='mobilenet_model_02.h5',
overwrite=True)
MobileNet Model Evaluation - training cycle #2¶
In [37]:
# Load the saved model
model_mobilenet_02.load_weights('mobilenet_model_02.h5')
# Evaluate the model on the hold out validation and test datasets
mn_val_eval_02 = model_mobilenet_02.evaluate_generator(valid_generator)
mn_test_eval_02 = model_mobilenet_02.evaluate_generator(test_generator)
print('Validation loss: {}'.format(mn_val_eval_02[0]))
print('Validation accuracy: {}'.format(mn_val_eval_02[1]))
print('Test loss: {}'.format(mn_test_eval_02[0]))
print('Test accuracy: {}'.format(mn_test_eval_02[1]))
# Prediction Probabilities
mn_predictions_02 = model_mobilenet_02.predict_generator(test_generator,steps = nb_samples, verbose=1)
# Prediction Labels
mn_pred_labels_02 = np.argmax(mn_predictions_02, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: MobileNet Training Cycle #2-----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, mn_pred_labels_02,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
In [50]:
Fine tune the model - training cycle #3¶
In [38]:
base_model_mobilenet_03 = MobileNet(input_shape=(224, 224, 3),
include_top = False)
x = base_model_mobilenet_03.output
flat = GlobalAveragePooling2D()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_mobilenet_03= Model(base_model_mobilenet_03.inputs, output)
model_mobilenet_03.load_weights('mobilenet_model_02_wt.h5')
# Call Backs
filepath = 'mobilenet_model_03_wt.h5'
es_03 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_03 = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
for layer in model_mobilenet_03.layers[:55]:
layer.trainable=False
model_mobilenet_03.compile(optimizer = Adam(lr = 0.0004),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
In [33]:
mobilenetv2_history_03 = model_mobilenet_03.fit_generator(train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es_03, cp_03],
validation_data = valid_generator)
In [34]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_mobilenet_03.save_weights(filepath='model_weights/mobilenet_model_03.h5', overwrite=True)
Model Evaluation - training cycle 3¶
In [39]:
# Load the saved model
model_mobilenet_03.load_weights('mobilenet_model_03.h5')
# Evaluate the model on the hold out validation and test datasets
mn_val_eval_03 = model_mobilenet_03.evaluate_generator(valid_generator)
mn_test_eval_03 = model_mobilenet_03.evaluate_generator(test_generator)
print('Validation loss: {}'.format(mn_val_eval_03[0]))
print('Validation accuracy: {}'.format(mn_val_eval_03[1]))
print('Test loss: {}'.format(mn_test_eval_03[0]))
print('Test accuracy: {}'.format(mn_test_eval_03[1]))
# Predict probability
mn_predictions_03 = model_mobilenet_03.predict_generator(test_generator,steps = nb_samples,verbose=1)
# Prediction labels
mn_pred_labels_03 = np.argmax(mn_predictions_03, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: MobileNet Training Cycle #3----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, mn_pred_labels_03,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
Fine tuning the entire model¶
In [40]:
base_model_mobilenet_04 = MobileNet(input_shape=(224, 224, 3),
include_top = False)
x = base_model_mobilenet_04.output
flat = GlobalAveragePooling2D()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_mobilenet_final= Model(base_model_mobilenet_04.inputs, output)
# Load weights from the previous traning session
model_mobilenet_final.load_weights('mobilenet_model_03_wt.h5')
# Call Backs
filepath = 'mobilenet_model_final_wt.h5'
es_04 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_04 = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
# For the final tuning of the entire let's use Stochastic Gradient Descent and slow tuning
sgd = SGD(lr=.00001, decay=1e-6, momentum=0.9, nesterov=True)
model_mobilenet_final.compile(optimizer = sgd,
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
In [40]:
mobilenetv2_history_final = model_mobilenet_final.fit_generator(train_generator,
steps_per_epoch=225,
epochs=50,
callbacks = [es_04, cp_04],
validation_data = valid_generator)
Model Evaluation - final training cycle¶
In [41]:
# Load the saved model
model_mobilenet_final.load_weights('mobilenet_model_final_wt.h5')
# Evaluate the model on the hold out validation and test datasets
mn_val_eval_final = model_mobilenet_final.evaluate_generator(valid_generator)
mn_test_eval_final = model_mobilenet_final.evaluate_generator(test_generator)
print('Validation loss: {}'.format(mn_val_eval_final[0]))
print('Validation accuracy: {}'.format(mn_val_eval_final[1]))
print('Test loss: {}'.format(mn_test_eval_final[0]))
print('Test accuracy: {}'.format(mn_test_eval_final[1]))
# Prediction probability
mn_predictions_final= model_mobilenet_final.predict_generator(test_generator,steps = nb_samples,verbose=1)
# Predict labels
mn_pred_labels_final = np.argmax(mn_predictions_final, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: MobileNet Training Cycle #3----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, mn_pred_labels_final,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
In [ ]:
## Confusion Matrices
In [42]:
mn_conf_mat_01 = pd.DataFrame(confusion_matrix(test_generator.classes, mn_pred_labels_01),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
mn_conf_mat_02 = pd.DataFrame(confusion_matrix(test_generator.classes, mn_pred_labels_02),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
mn_conf_mat_03 = pd.DataFrame(confusion_matrix(test_generator.classes, mn_pred_labels_03),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
mn_conf_mat_final = pd.DataFrame(confusion_matrix(test_generator.classes, mn_pred_labels_final),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
In [43]:
# Plot confusion matrices
sns.set(font_scale=1.8)
fig, ([ax1,ax2],[ax3,ax4]) = plt.subplots(nrows=2, ncols=2, figsize=(28,28))
#ax1
sns.heatmap(mn_conf_mat_01, ax=ax1, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax1.set_ylabel("Actual Label", fontsize=24)
ax1.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="MobileNet Training Round #1 \nAccuracy Score: {0:.3f}".format(mn_test_eval_01[1])
ax1.set_title(all_sample_title, size=32)
ax1.set_ylim(len(mn_conf_mat_01)-0.1, -0.1)
#ax2
sns.heatmap(mn_conf_mat_02, ax=ax2, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax2.set_ylabel("Actual Label", fontsize=24)
ax2.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="MobileNet Training Round #2 \nAccuracy Score: {0:.3f}".format(mn_test_eval_02[1])
ax2.set_title(all_sample_title, size=32)
ax2.set_ylim(len(mn_conf_mat_02)-0.1, -0.1)
#ax3
sns.heatmap(mn_conf_mat_03, ax=ax3, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax3.set_ylabel("Actual Label", fontsize=24)
ax3.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="MobileNet Training Round#3 \nAccuracy Score: {0:.3f}".format(mn_test_eval_03[1])
ax3.set_title(all_sample_title, size=32)
ax3.set_ylim(len(mn_conf_mat_03)-0.1, -0.1)
#ax4
sns.heatmap(mn_conf_mat_final, ax=ax4, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax4.set_ylabel("Actual Label", fontsize=24)
ax4.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="MobileNet Training Round#4 \nAccuracy Score: {0:.3f}".format(mn_test_eval_final[1])
ax4.set_title(all_sample_title, size=32)
ax4.set_ylim(len(mn_conf_mat_final)-0.1, -0.1)
plt.tight_layout()
Resent50¶
Training cycle #1¶
In [44]:
base_model_resnet50 = ResNet50(input_shape=(224,224,3),
include_top=False,
weights='imagenet')
x = base_model_resnet50.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_resnet50_01 = Model(base_model_resnet50.inputs, output)
# Call Backs
filepath = 'resent50_base_model_wt.h5'
es = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
# compile the model
model_resnet50_01.compile(loss="categorical_crossentropy", optimizer=Adam(lr = 0.0004), metrics=["accuracy"])
In [15]:
resnet50_history_01 = model_resnet50_01.fit_generator(train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es, cp],
validation_data = valid_generator)
In [16]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_resnet50_01.save_weights(filepath='resent50_base_model_wt.h5', overwrite=True)
ResNet50 Model Evaluation - training cycle #1¶
In [46]:
# Load the saved model
model_resnet50_01.load_weights('resent50_base_model_wt.h5')
# Evaluate the model on the hold out validation and test datasets
res_val_eval_01 = model_resnet50_01.evaluate_generator(valid_generator)
res_test_eval_01 = model_resnet50_01.evaluate_generator(test_generator)
print('Validation loss: {}'.format(res_val_eval_01[0]))
print('Validation accuracy: {}'.format(res_val_eval_01[1]))
print('Test loss: {}'.format(res_test_eval_01[0]))
print('Test accuracy: {}'.format(res_test_eval_01[1]))
# Predict probabilities
res_predictions_01 = model_resnet50_01.predict_generator(test_generator,steps = nb_samples,verbose=1)
# Predict labels
res_pred_labels_01 = np.argmax(res_predictions_01, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: ReseNet50 Training Cycle #1----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, res_pred_labels_01,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
Fine tuning ResNet50 model - training cycle #2¶
In [47]:
base_model_resnet50_02 = ResNet50(input_shape=(224, 224, 3),
include_top = False)
x = base_model_resnet50_02.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_resnet50_02 = Model(base_model_resnet50_02.inputs, output)
model_resnet50_02.load_weights('resent50_base_model_wt.h5')
for layer in model_resnet50_02.layers[:160]:
layer.trainable= False
# Call Backs
filepath_02 = 'resent50_model_02_wt.h5'
es_02 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_02 = ModelCheckpoint(filepath, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
model_resnet50_02.compile(loss="categorical_crossentropy", optimizer=Adam(lr = 0.0004), metrics=["accuracy"])
In [23]:
resnet50_history_02 = model_resnet50_02.fit_generator(train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es_02, cp_02],
validation_data = valid_generator)
In [25]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_resnet50_02.save_weights(filepath='resent50_model_02_wt.h5', overwrite=True)
Model Evaluation - training cycle #2¶
In [48]:
# Load the saved model
model_resnet50_02.load_weights('resent50_model_02_wt.h5')
# Evaluate the model on the hold out validation and test datasets
res_val_eval_02 = model_resnet50_02.evaluate_generator(valid_generator)
res_test_eval_02 = model_resnet50_02.evaluate_generator(test_generator)
print('Validation loss: {}'.format(res_val_eval_02[0]))
print('Validation accuracy: {}'.format(res_val_eval_02[1]))
print('Test loss: {}'.format(res_test_eval_02[0]))
print('Test accuracy: {}'.format(res_test_eval_02[1]))
print('*'*75)
# Predict probabilities
res_predictions_02 = model_resnet50_02.predict_generator(test_generator, steps = nb_samples,verbose=1)
# Predict labels
res_pred_labels_02 = np.argmax(res_predictions_02, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: ReseNet50 Training Cycle #2----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, res_pred_labels_02,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
Fine tuning model # Round 3¶
In [49]:
base_model_resnet50_03 = ResNet50(input_shape=(224, 224, 3),
include_top = False)
x = base_model_resnet50_03.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_resnet50_03 = Model(base_model_resnet50_03.inputs, output)
model_resnet50_03.load_weights('resent50_model_02_wt.h5')
for layer in model_resnet50_03.layers[:118]:
layer.trainable= False
# Call Backs
filepath_03 = 'resent50_model_03_wt.h5'
es_03 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_03 = ModelCheckpoint(filepath_03, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
model_resnet50_03.compile(loss="categorical_crossentropy", optimizer=Adam(lr = 0.0004), metrics=["accuracy"])
In [30]:
resnet50_history_03 = model_resnet50_03.fit_generator(train_generator,
steps_per_epoch=225,
epochs=10,
callbacks = [es_02, cp_02],
validation_data = valid_generator)
In [ ]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_resnet50_03.save_weights(filepath='resent50_model_03_wt.h5', overwrite=True)
ResNet50 Model Evaluation - training cyle #3¶
In [50]:
# Load the saved model
model_resnet50_03.load_weights('resent50_model_03_wt.h5')
# Evaluate the model on the hold out validation and test datasets
res_val_eval_03 = model_resnet50_03.evaluate_generator(valid_generator)
res_test_eval_03 = model_resnet50_03.evaluate_generator(test_generator)
print('Validation loss: {0:.3f}'.format(res_val_eval_03[0]))
print('Validation accuracy: {0:.3f}'.format(res_val_eval_03[1]))
print('Test loss: {0:.3f}'.format(res_test_eval_03[0]))
print('Test accuracy: {0:.3f}'.format(res_test_eval_03[1]))
# Predict probabilities
res_predictions_03 = model_resnet50_03.predict_generator(test_generator,steps = nb_samples,verbose=1)
# Predict labels
res_pred_labels_03 = np.argmax(res_predictions_03, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: ReseNet50 Training Cycle #3----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, res_pred_labels_03,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
Fine tuning Resnet50 model training cycle #4¶
In [51]:
base_model_resnet50_04 = ResNet50(input_shape=(224, 224, 3),
include_top = False)
x = base_model_resnet50_04.output
flat = Flatten()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_resnet50_04 = Model(base_model_resnet50_04.inputs, output)
model_resnet50_04.load_weights('resent50_model_03_wt.h5')
# Call Backs
filepath_04 = 'resent50_model_04_wt.h5'
es_04 = EarlyStopping(monitor='loss', verbose=1, mode='min', patience=4)
cp_04 = ModelCheckpoint(filepath_04, monitor='val_loss',verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
model_resnet50_04.compile(loss="categorical_crossentropy", optimizer=Adam(lr = 0.0004), metrics=["accuracy"])
In [35]:
resnet50_history_04 = model_resnet50_04.fit_generator(train_generator,
steps_per_epoch=225,
epochs=30,
callbacks = [es_04, cp_04],
validation_data = valid_generator)
In [ ]:
# save model
if not os.path.isdir('model_weights/'):
os.mkdir('model_weights/')
model_resnet50_04.save_weights(filepath='model_weights/resent50_model_04_wt.h5', overwrite=True)
In [52]:
# Load the saved model
model_resnet50_04.load_weights('resent50_model_04_wt.h5')
# Evaluate the model on the hold out validation and test datasets
res_val_eval_04 = model_resnet50_04.evaluate_generator(valid_generator)
res_test_eval_04 = model_resnet50_04.evaluate_generator(test_generator)
print('Validation loss: {}'.format(res_val_eval_04[0]))
print('Validation accuracy: {}'.format(res_val_eval_04[1]))
print('Test loss: {}'.format(res_test_eval_04[0]))
print('Test accuracy: {}'.format(res_test_eval_04[1]))
res_predictions_04 = model_resnet50_04.predict_generator(test_generator,
steps = nb_samples,
verbose=1)
res_pred_labels_04 = np.argmax(res_predictions_04, axis=1)
# Classification Report
print('|'+'-'*67+'|')
print('|-------Classification Report: ReseNet50 Training Cycle #4----------|')
print('|'+'-'*67+'|')
print(classification_report(test_generator.classes, res_pred_labels_04,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
In [53]:
# Confusion Matrices
res_conf_mat_01 = pd.DataFrame(confusion_matrix(test_generator.classes, res_pred_labels_01),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
res_conf_mat_02 = pd.DataFrame(confusion_matrix(test_generator.classes, res_pred_labels_02),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
res_conf_mat_03 = pd.DataFrame(confusion_matrix(test_generator.classes, res_pred_labels_03),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
res_conf_mat_04 = pd.DataFrame(confusion_matrix(test_generator.classes, res_pred_labels_04),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
In [54]:
# Plot Confusion Matrices
sns.set(font_scale=1.8)
fig, ([ax1,ax2],[ax3,ax4]) = plt.subplots(nrows=2, ncols=2, figsize=(28,28))
#ax1
sns.heatmap(res_conf_mat_01, ax=ax1, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax1.set_ylabel("Actual Label", fontsize=24)
ax1.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="ResNet50 Training Round #1 \nAccuracy Score: {0:.3f}".format(res_test_eval_01[1])
ax1.set_title(all_sample_title, size=32)
ax1.set_ylim(len(res_conf_mat_01)-0.1, -0.1)
#ax2
sns.heatmap(res_conf_mat_02, ax=ax2, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax2.set_ylabel("Actual Label", fontsize=24)
ax2.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="ResNet50 Training Round #2 \nAccuracy Score: {0:.3f}".format(res_test_eval_02[1])
ax2.set_title(all_sample_title, size=32)
ax2.set_ylim(len(res_conf_mat_02)-0.1, -0.1)
#ax3
sns.heatmap(res_conf_mat_03, ax=ax3, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax3.set_ylabel("Actual Label", fontsize=24)
ax3.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="ResNet50 Training Round#3 \nAccuracy Score: {0:.3f}".format(res_test_eval_03[1])
ax3.set_title(all_sample_title, size=32)
ax3.set_ylim(len(res_conf_mat_03)-0.1, -0.1)
#ax4
sns.heatmap(res_conf_mat_04, ax=ax4, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 18})
ax4.set_ylabel("Actual Label", fontsize=24)
ax4.set_xlabel("Predicted Label", fontsize=24)
all_sample_title="ResNet50 Training Round#4 \nAccuracy Score: {0:.3f}".format(res_test_eval_04[1])
ax4.set_title(all_sample_title, size=32)
ax4.set_ylim(len(res_conf_mat_04)-0.1, -0.1)
plt.tight_layout()
In [89]:
# Test data
filenames = test_generator.filenames
nb_samples = len(filenames)
class_labels = test_generator.class_indices
class_names = {value:key for key,value in class_labels.items()}
labels = (ind_test_generator.class_indices)
labels = dict((v,k) for k,v in labels.items())
predictions = [labels[k] for k in res_pred_labels_04]
filenames = test_generator.filenames
results = pd.DataFrame({"Filename":filenames,
"Predictions":predictions})
random_files = results.sample(36)
filenames = random_files['Filename'].tolist()
predicted_labels = random_files['Predictions'].tolist()
test_file_paths = ['/floyd/home/output/test/'+ filename for filename in filenames]
In [87]:
# Tissue types dictionary mapping class names with full names
tissue_types = {"ADI": "Adipose tissue",
"BACK": "Background",
"DEB": "Debris",
"LYM": "Lymphocyte aggregates",
"MUC": "Mucus",
"MUS": "Muscle",
"NORM": "Normal mucosa",
"STR": "Stroma",
"TUM": "Tumor epithelium"}
fig = plt.figure(figsize=(45,60))
fig.subplots_adjust(top=0.88)
columns = 4
rows = 9
for i in range(1, columns*rows+1):
fig.add_subplot(rows, columns, i)
plt.imshow(mpimg.imread(test_file_paths[i-1]))
plt.axis('off')
true_label = test_file_paths[i-1].split('/')[-2]
predicted_label = predicted_labels[i-1]
plt.title("True Label: {}\nPredicted Label: {}".format(tissue_types[true_label], tissue_types[predicted_label]), fontsize=28)
plt.tight_layout()
plt.show()
Evaluating the CNN model on an independent dataset¶
In [55]:
# Define a path to the independent
ind_test_dir = os.path.join(base_dir, "CRC-VAL-HE-7K")
In [56]:
# Image generator
ind_test_generator = test_datagen.flow_from_directory(ind_test_dir,
batch_size=1,
target_size=(224,224),
class_mode='categorical',
shuffle=False,
seed=42,
color_mode='rgb')
# File names and number of images
filenames = ind_test_generator.filenames
nb_samples = len(filenames)
Performance of the VGG19 model on the Independent Dataset¶
In [57]:
vgg_val_eval_7K = model_vgg19_03.evaluate_generator(ind_test_generator)
print('Loss: {0:.4f}'.format(vgg_val_eval_7K[0]))
print('Accuracy: {0:.4f}'.format(vgg_val_eval_7K[1]))
# Predict Probabilities
vgg_predictions_7K = model_vgg19_03.predict_generator(ind_test_generator, steps = nb_samples, verbose=1)
# Predict Labels
vgg_pred_labels_7K = np.argmax(vgg_predictions_7K, axis=1)
# Classification Report
print('|----------------------------------------------------------|')
print('|----Independent_Data--Classification Report-(VGG19)-------|')
print('|----------------------------------------------------------|')
print(classification_report(ind_test_generator.classes, vgg_pred_labels_7K,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
# Confusion Matrix
vgg_conf_mat_7K = pd.DataFrame(confusion_matrix(ind_test_generator.classes, vgg_pred_labels_7K),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
Performance of the MobileNet model on the Independent Dataset¶
In [58]:
mobilenet_val_eval_7K = model_mobilenet_final.evaluate_generator(ind_test_generator)
print('Loss: {}'.format(mobilenet_val_eval_7K[0]))
print('Accuracy: {}'.format(mobilenet_val_eval_7K[1]))
# Predict probabilities
mobilenet_predictions_7K = model_mobilenet_03.predict_generator(ind_test_generator, steps = nb_samples, verbose=1)
# Predict labels
mobilenet_pred_labels_7K = np.argmax(mobilenet_predictions_7K, axis=1)
# Classification Report
print('----------------------------------------------------------')
print('---Independent_Data--Classification Report-(MobileNet)----')
print('----------------------------------------------------------')
print(classification_report(ind_test_generator.classes, mobilenet_pred_labels_7K,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
# Confusion Matrix
mobilenet_conf_mat_7K = pd.DataFrame(confusion_matrix(ind_test_generator.classes, mobilenet_pred_labels_7K),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
Performance of the ResNet50 model on the Independent Dataset¶
In [61]:
# Load the saved model
model_resnet50_04.load_weights('resent50_model_04_wt.h5')
# Evaluate the model on the independent dataset
resnet50_ind_eval_7K = model_resnet50_04.evaluate_generator(ind_test_generator)
print('Loss: {}'.format(resnet50_ind_eval_7K[0]))
print('Accuracy: {}'.format(resnet50_ind_eval_7K[1]))
# Predict probabilities
resnet50_predictions_7K = model_resnet50_04.predict_generator(ind_test_generator, steps = nb_samples, verbose=1)
# Predict labels
resnet50_pred_labels_7K = np.argmax(resnet50_predictions_7K, axis=1)
# Classification Report
print('----------------------------------------------------------')
print('---Independent_Data--Classification Report-(MobileNet)----')
print('----------------------------------------------------------')
print(classification_report(ind_test_generator.classes, resnet50_pred_labels_7K,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
# Confusion Matrix
resent50_conf_mat_7K = pd.DataFrame(confusion_matrix(ind_test_generator.classes, resnet50_pred_labels_7K),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
In [67]:
# Plot confusion matrices
sns.set(font_scale=1.8)
fig, ([ax1,ax2,ax3]) = plt.subplots(nrows=3, ncols=1, figsize=(20,28))
#ax1
sns.heatmap(vgg_conf_mat_7K, ax=ax1, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 14})
ax1.set_ylabel("Actual Label", fontsize=20)
ax1.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="VGG19 Performance on Independent Dataset\nAccuracy Score: {0:.3f}".format(vgg_val_eval_7K[1])
ax1.set_title(all_sample_title, size=20)
ax1.set_ylim(len(vgg_conf_mat_7K)-0.1, -0.1)
#ax2
sns.heatmap(mobilenet_conf_mat_7K, ax=ax2, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 14})
ax2.set_ylabel("Actual Label", fontsize=20)
ax2.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="MobileNet Performance on Independent Dataset\nAccuracy Score: {0:.3f}".format(mobilenet_val_eval_7K[1])
ax2.set_title(all_sample_title, size=20)
ax2.set_ylim(len(mobilenet_conf_mat_7K)-0.1, -0.1)
#ax3
sns.heatmap(resent50_conf_mat_7K, ax=ax3, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 14})
ax3.set_ylabel("Actual Label", fontsize=20)
ax3.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="ResNet50 Performance on Independent Dataset\nAccuracy Score: {0:.3f}".format(resnet50_ind_eval_7K[1])
ax3.set_title(all_sample_title, size=20)
ax3.set_ylim(len(resent50_conf_mat_7K)-0.1, -0.1)
fig.tight_layout()
Model Stacking¶
In [68]:
# Load tuned VGG19 model
VGG19_model = load_model("vgg19_model_finetuned.h5")
# Load tuned ResNet50 model
ResNet50_model = load_model("resent50_model_04_wt.h5")
# Load tuned MobileNet model
base_model_mobilenet_03 = MobileNet(input_shape=(224, 224, 3),
include_top = False)
x = base_model_mobilenet_03.output
flat = GlobalAveragePooling2D()(x)
hidden_1 = Dense(1024, activation='relu')(flat)
drop_1 = Dropout(0.2)(hidden_1)
hidden_2 = Dense(512, activation='relu')(drop_1)
drop_2 = Dropout(0.3)(hidden_2)
output = Dense(9, activation= 'softmax')(drop_2)
model_mobilenet_03= Model(base_model_mobilenet_03.inputs, output)
model_mobilenet_03.compile(loss="categorical_crossentropy", optimizer=Adam(lr=0.0001), metrics=["accuracy"])
model_mobilenet_03.load_weights("mobilenet_model_final_wt.h5")
In [69]:
# get prediction probablilities from all three models
vgg_predictions_7K = VGG19_model.predict_generator(ind_test_generator,steps = nb_samples,verbose=1)
resnet_predictions_7K = ResNet50_model.predict_generator(ind_test_generator,steps = nb_samples,verbose=1)
mobilenet_predictions_7K = model_mobilenet_03.predict_generator(ind_test_generator,steps = nb_samples,verbose=1)
In [70]:
# Average prediction of all three models
combined_prediction = np.mean([vgg_predictions_7K, resnet_predictions_7K, mobilenet_predictions_7K], axis=0)
combined_prediction_labels = np.argmax(combined_prediction, axis=1)
In [74]:
# Classification Report
print('----------------------------------------------------------')
print('-Independent_Data--Classification Report-(Combined Model)-')
print('----------------------------------------------------------')
print(classification_report(ind_test_generator.classes, combined_prediction_labels,
target_names=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM']))
In [73]:
# Confusion Matrix
combined_conf_mat_7K = pd.DataFrame(confusion_matrix(ind_test_generator.classes, combined_prediction_labels),
index=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'],
columns=['ADI', 'BACK', 'DEB', 'LYM', 'MUC', 'MUS', 'NORM', 'STR', 'TUM'])
# Plot confusion matrices
sns.set(font_scale=1.8)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(12,12))
#ax1
sns.heatmap(combined_conf_mat_7K, ax=ax, annot=True, fmt=".1f", linewidths=0.5, square=True, cmap='Blues_r', annot_kws={"size": 14})
ax.set_ylabel("Actual Label", fontsize=20)
ax.set_xlabel("Predicted Label", fontsize=20)
all_sample_title="Combined Performance on Independent Dataset"
ax.set_title(all_sample_title, size=20)
ax.set_ylim(len(vgg_conf_mat_7K)-0.1, -0.1)
Out[73]:
In [ ]: